{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "# from fastai import *\n",
    "# from fastai.text import *\n",
    "import kenlm\n",
    "from tqdm import tqdm\n",
    "import fastText\n",
    "import pandas as pd\n",
    "from bleu import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "# #loading openai GPT\n",
    "\n",
    "# import torch, os\n",
    "# from pytorch_pretrained_bert import OpenAIGPTTokenizer, OpenAIGPTLMHeadModel\n",
    "# import numpy as np\n",
    "\n",
    "# lm_model_special_tokens = [\"<POS>\",\"<NEG>\",\"<END>\"]\n",
    "# lm_tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt', special_tokens=lm_model_special_tokens)\n",
    "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "# lm_model = OpenAIGPTLMHeadModel.from_pretrained('openai-gpt', num_special_tokens=len(lm_model_special_tokens))\n",
    "\n",
    "# path = os.path.join(os.getcwd(), \"./log_yelp_lm/yelp_language_model_2.bin\")\n",
    "# lm_model_state_dict = torch.load(path)\n",
    "# lm_model.load_state_dict(lm_model_state_dict)\n",
    "# lm_model.to(device)\n",
    "# lm_model.eval()\n",
    "\n",
    "# lm_loss = torch.nn.CrossEntropyLoss(ignore_index=-1, reduction='none')\n",
    "\n",
    "# def calculate_ppl_gpt(sentence_batch, sentiment):\n",
    "#     # tokenize the sentences\n",
    "#     tokenized_ids = [None for i in range(len(sentence_batch))]\n",
    "    \n",
    "#     for i in range(len(sentence_batch)):\n",
    "#         tkns = lm_tokenizer.tokenize(sentiment + ' ' +sentence_batch[i])\n",
    "#         tokenized_ids[i] = lm_tokenizer.convert_tokens_to_ids(tkns)\n",
    "#     sen_lengths = [len(x) for x in tokenized_ids]\n",
    "#     max_sen_lenght = max(sen_lengths)\n",
    "    \n",
    "#     n_batch = len(sentence_batch)\n",
    "#     input_ids = np.zeros( shape=(n_batch, max_sen_lenght), dtype=np.int64)\n",
    "#     lm_labels = np.full(shape=(n_batch, max_sen_lenght), fill_value=-1)\n",
    "    \n",
    "#     for i, tokens in enumerate(tokenized_ids):\n",
    "#         input_ids[i, :len(tokens)] = tokens\n",
    "#         lm_labels[i, :len(tokens)] = tokens[1:] + [lm_tokenizer.special_tokens[\"<END>\"]]\n",
    "    \n",
    "#     input_ids = torch.tensor(input_ids).to(device)\n",
    "#     lm_labels = torch.tensor(lm_labels).to(device)\n",
    "#     with torch.no_grad():\n",
    "#         lm_pred = lm_model(input_ids)\n",
    "#     loss_val = lm_loss(lm_pred.view(-1, lm_pred.size(-1)), lm_labels.view(-1))\n",
    "#     normalized_loss = loss_val.view(n_batch,-1).sum(dim= -1) / torch.tensor(sen_lengths, dtype=torch.float32).to(device)\n",
    "#     #normalized_loss = loss_val.view(n_batch,-1).sum(dim= -1)\n",
    "#     ppl = torch.exp(normalized_loss)\n",
    "#     return  ppl.tolist() \n",
    "\t\n",
    "# def calculate_dataset_ppl(input_sentences,correct_sentiment,bs=16):\n",
    "#     ppl = []\n",
    "#     steps = len(input_sentences) // bs\n",
    "#     for i in range(steps + 1):\n",
    "#         if i != steps:\n",
    "#             inp = input_sentences[i * bs : i * bs + bs]\n",
    "#             ppl.extend(calculate_ppl(inp, sentiment=sentiment))\n",
    "#         else:\n",
    "#             inp = input_sentences[i * bs: len(input_sentences)]\n",
    "#             ppl.extend(calculate_ppl(inp, sentiment=sentiment))\n",
    "#     return sum(ppl) / len(ppl)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bert classifier\n",
    "import torch, os\n",
    "from tqdm import trange\n",
    "\n",
    "from pytorch_pretrained_bert.file_utils import PYTORCH_PRETRAINED_BERT_CACHE\n",
    "from pytorch_pretrained_bert.modeling import BertForSequenceClassification, BertConfig, WEIGHTS_NAME, CONFIG_NAME\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "\n",
    "model_cls = BertForSequenceClassification.from_pretrained(\"./bert_classifier/yelp\", num_labels=2)\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
    "\n",
    "model_cls.to('cuda')\n",
    "model_cls.eval()\n",
    "\n",
    "max_seq_len=70\n",
    "sm = torch.nn.Softmax(dim=-1)\n",
    "\n",
    "def evaluate_dev_set(input_sentences, labels, bs=32):\n",
    "    \"\"\"\n",
    "    To evaluate whole dataset and return accuracy\n",
    "    \"\"\"\n",
    "    ids = []\n",
    "    segment_ids = []\n",
    "    input_masks = []\n",
    "    pred_lt = []\n",
    "    for sen in input_sentences:\n",
    "        text_tokens = tokenizer.tokenize(sen)\n",
    "        tokens = [\"[CLS]\"] + text_tokens + [\"[SEP]\"]\n",
    "        temp_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "        input_mask = [1] * len(temp_ids)\n",
    "        segment_id = [0] * len(temp_ids)\n",
    "        padding = [0] * (max_seq_len - len(temp_ids))\n",
    "\n",
    "        temp_ids += padding\n",
    "        input_mask += padding\n",
    "        segment_id += padding\n",
    "        \n",
    "        ids.append(temp_ids)\n",
    "        input_masks.append(input_mask)\n",
    "        segment_ids.append(segment_id)\n",
    "    \n",
    "    ids = torch.tensor(ids).to('cuda')\n",
    "    segment_ids = torch.tensor(segment_ids).to('cuda')\n",
    "    input_masks = torch.tensor(input_masks).to('cuda')\n",
    "    \n",
    "    steps = len(ids) // bs\n",
    "    \n",
    "    for i in trange(steps+1):\n",
    "        if i == steps:\n",
    "            temp_ids = ids[i * bs : len(ids)]\n",
    "            temp_segment_ids = segment_ids[i * bs: len(ids)]\n",
    "            temp_input_masks = input_masks[i * bs: len(ids)]\n",
    "        else:\n",
    "            temp_ids = ids[i * bs : i * bs + bs]\n",
    "            temp_segment_ids = segment_ids[i * bs: i * bs + bs]\n",
    "            temp_input_masks = input_masks[i * bs: i * bs + bs]\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            preds = sm(model_cls(temp_ids, temp_segment_ids, temp_input_masks))\n",
    "        \n",
    "        #preds = preds.view(-1,bs)\n",
    "        try:\n",
    "            args = torch.argmax(preds, dim=-1)\n",
    "            pred_lt.extend(args.tolist())\n",
    "        except RuntimeError:\n",
    "            pass\n",
    "    accuracy = sum(np.array(pred_lt) == np.array(labels)) / len(labels)\n",
    "    \n",
    "    return accuracy, pred_lt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_pretrained_bert.tokenization_gpt2:loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json from cache at /home/ubuntu/.pytorch_pretrained_bert/f2808208f9bec2320371a9f5f891c184ae0b674ef866b79c58177067d15732dd.1512018be4ba4e8726e41b9145129dc30651ea4fec86aa61f4b9f40bf94eac71\n",
      "INFO:pytorch_pretrained_bert.tokenization_gpt2:loading merges file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt from cache at /home/ubuntu/.pytorch_pretrained_bert/d629f792e430b3c76a1291bb2766b0a047e36fae0588f9dbc1ae51decdff691b.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
      "INFO:pytorch_pretrained_bert.modeling_gpt2:loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin from cache at /home/ubuntu/.pytorch_pretrained_bert/4295d67f022061768f4adc386234dbdb781c814c39662dd1662221c309962c55.778cf36f5c4e5d94c8cd9cefcf2a580c8643570eb327f0d4a1f007fab2acbdf1\n",
      "INFO:pytorch_pretrained_bert.modeling_gpt2:loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json from cache at /home/ubuntu/.pytorch_pretrained_bert/4be02c5697d91738003fb1685c9872f284166aa32e061576bbe6aaeb95649fcf.085d5f6a8e7812ea05ff0e6ed0645ab2e75d80387ad55c1ad9806ee70d272f80\n",
      "INFO:pytorch_pretrained_bert.modeling_gpt2:Model config {\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"n_ctx\": 1024,\n",
      "  \"n_embd\": 768,\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"n_positions\": 1024,\n",
      "  \"vocab_size\": 50257\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# import torch, os\n",
    "# from pytorch_pretrained_bert import GPT2Tokenizer, GPT2Model, GPT2LMHeadModel\n",
    "\n",
    "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "# import logging\n",
    "# logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "# gpt2_lm_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "# gpt2_lm_model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
    "# path = os.path.join(os.getcwd(), \"./GPT2/yelp_language_model_1.bin\")\n",
    "# gpt2_lm_model_state_dict = torch.load(path)\n",
    "# gpt2_lm_model.load_state_dict(gpt2_lm_model_state_dict)\n",
    "# gpt2_lm_model.to(device)\n",
    "# gpt2_lm_model.eval()\n",
    "\n",
    "# lm_loss = torch.nn.CrossEntropyLoss(ignore_index=-1, reduction='none')\n",
    "# def calculate_ppl_gpt2(sentence_batch):\n",
    "#     # tokenize the sentences\n",
    "#     tokenized_ids = [None for i in range(len(sentence_batch))]\n",
    "    \n",
    "#     for i in range(len(sentence_batch)):\n",
    "#         tokenized_ids[i] = gpt2_lm_tokenizer.encode(sentence_batch[i])\n",
    "        \n",
    "#     sen_lengths = [len(x) for x in tokenized_ids]\n",
    "#     max_sen_lenght = max(sen_lengths)\n",
    "    \n",
    "#     n_batch = len(sentence_batch)\n",
    "#     input_ids = np.zeros( shape=(n_batch, max_sen_lenght), dtype=np.int64)\n",
    "#     lm_labels = np.full(shape=(n_batch, max_sen_lenght), fill_value=-1)\n",
    "    \n",
    "#     for i, tokens in enumerate(tokenized_ids):\n",
    "#         input_ids[i, :len(tokens)] = tokens\n",
    "#         lm_labels[i, :len(tokens)-1] = tokens[1:] \n",
    "    \n",
    "#     input_ids = torch.tensor(input_ids).to(device)\n",
    "#     lm_labels = torch.tensor(lm_labels).to(device)\n",
    "#     with torch.no_grad():\n",
    "#         lm_pred = gpt2_lm_model(input_ids)\n",
    "#     loss_val = lm_loss(lm_pred[0].view(-1, lm_pred[0].size(-1)), lm_labels.view(-1))\n",
    "#     normalized_loss = loss_val.view(n_batch,-1).sum(dim= -1) / torch.tensor(sen_lengths, dtype=torch.float32).to(device)\n",
    "#     #normalized_loss = loss_val.view(n_batch,-1).sum(dim= -1)\n",
    "#     ppl = torch.exp(normalized_loss)\n",
    "#     return  ppl.tolist() \n",
    "\n",
    "from pytorch_pretrained_bert import GPT2Tokenizer, GPT2Model, GPT2LMHeadModel\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "import logging\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "lm_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "lm_model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
    "path = os.path.join(os.getcwd(), \"GPT2/yelp_language_model_1.bin\")\n",
    "lm_model_state_dict = torch.load(path)\n",
    "lm_model.load_state_dict(lm_model_state_dict)\n",
    "lm_model.to(device)\n",
    "lm_model.eval()\n",
    "\n",
    "lm_loss = torch.nn.CrossEntropyLoss(ignore_index=-1, reduction='none')\n",
    "\n",
    "\n",
    "def calculate_ppl_gpt2(sentence_batch, bs=16):\n",
    "    # tokenize the sentences\n",
    "    tokenized_ids = [None for i in range(len(sentence_batch))]\n",
    "    ppl = [None for i in range(len(sentence_batch))]\n",
    "    \n",
    "    for i in range(len(sentence_batch)):\n",
    "        tokenized_ids[i] = lm_tokenizer.encode(sentence_batch[i])\n",
    "        \n",
    "    sen_lengths = [len(x) for x in tokenized_ids]\n",
    "    max_sen_length = max(sen_lengths)\n",
    "    \n",
    "    n_batch = len(sentence_batch)\n",
    "    input_ids = np.zeros( shape=(n_batch, max_sen_length), dtype=np.int64)\n",
    "    lm_labels = np.full(shape=(n_batch, max_sen_length), fill_value=-1)\n",
    "    \n",
    "    for i, tokens in enumerate(tokenized_ids):\n",
    "        input_ids[i, :len(tokens)] = tokens\n",
    "        lm_labels[i, :len(tokens)-1] = tokens[1:] \n",
    "    \n",
    "    input_ids = torch.tensor(input_ids)#.to(device)\n",
    "    lm_labels = torch.tensor(lm_labels)#.to(device)\n",
    "    \n",
    "    steps = n_batch // bs\n",
    "    \n",
    "    for i in range(steps+1):\n",
    "        \n",
    "        if i == steps:\n",
    "            temp_input_ids = input_ids[i * bs : n_batch]\n",
    "            temp_lm_labels = lm_labels[i * bs : n_batch]\n",
    "            temp_sen_lengths = sen_lengths[i * bs : n_batch]\n",
    "        else:\n",
    "            temp_input_ids = input_ids[i * bs : i * bs + bs]\n",
    "            temp_lm_labels = lm_labels[i * bs : i * bs + bs]\n",
    "            temp_sen_lengths = sen_lengths[i * bs : i * bs + bs]\n",
    "            \n",
    "        temp_input_ids = temp_input_ids.to('cuda')\n",
    "        temp_lm_labels = temp_lm_labels.to('cuda')\n",
    "            \n",
    "        with torch.no_grad():\n",
    "            lm_pred = lm_model(temp_input_ids)\n",
    "            \n",
    "        loss_val = lm_loss(lm_pred[0].view(-1, lm_pred[0].size(-1)), temp_lm_labels.view(-1))\n",
    "        normalized_loss = loss_val.view(len(temp_input_ids),-1).sum(dim= -1) / torch.tensor(temp_sen_lengths, dtype=torch.float32).to(device)\n",
    "        tmp_ppl = torch.exp(normalized_loss)\n",
    "        ppl[i * bs: i * bs + len(temp_input_ids)] = tmp_ppl.tolist()\n",
    "    \n",
    "    return  ppl\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[116.72223663330078, 81.89402770996094]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_ppl_gpt2([\"totally random lol\", \"this is a good sentance\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LOADING MODELS\n",
    "\n",
    "# #fastai classifier \n",
    "# path = Path('fastaimodels')\n",
    "\n",
    "# data_clas = load_data(path, 'data_clas.pkl', bs=48)\n",
    "# fastai_classifier = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)\n",
    "# fastai_classifier.load_encoder('yelp_fine_tuned_enc')\n",
    "# fastai_classifier.load('yelp_classifier')\n",
    "\n",
    "# #fastai lm\n",
    "# data_lm = load_data(path, 'data_lm.pkl', bs=48)\n",
    "# fastai_lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3)\n",
    "# fastai_lm.load_encoder('yelp_fine_tuned_enc')\n",
    "# fastai_lm.load('yelp_fine_tuned')\n",
    "\n",
    "#fasttext classifier\n",
    "classifier_model = fastText.load_model('fasttextmodel/model_yelp.bin')\n",
    "\n",
    "#kenlm lm\n",
    "kenlm_lm = kenlm.Model('kenlmmodel/yelp.arpa')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # fastai lm check\n",
    "# print(\"\\n\".join(fastai_lm.predict(\"hey man\", 20, temperature=0.75) for _ in range(2)))\n",
    "\n",
    "# # fastai ppl functions\n",
    "# def perplexitylm(sentance):\n",
    "#     tokens = sentance.split()\n",
    "#     str_list = []\n",
    "#     for i, x in enumerate(tokens):\n",
    "#         str1 = \" \".join(tokens[:i+1])\n",
    "#         str_list.append(str1)\n",
    "#     prob = []\n",
    "\n",
    "#     for i in range(0, len(str_list)-1):\n",
    "#         xb, yb = data_lm.one_item(str_list[i])\n",
    "#         #print(xb, yb)\n",
    "#         p1=fastai_lm.pred_batch(batch=(xb,yb))[0][-1]\n",
    "#         prob.append(p1[data_lm.one_item(tokens[i+1])[0][0][-1].item()].item())\n",
    "    \n",
    "#     perplexity = torch.exp(-torch.mean(torch.log(torch.tensor(prob)))).item()\n",
    "    \n",
    "#     return perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/25 [00:00<?, ?it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:05,  2.83it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:04,  3.06it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:04,  3.17it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.23it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.27it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:03,  3.31it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:02<00:02,  3.34it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.37it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:02,  3.38it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.40it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.41it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.42it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.43it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:04<00:00,  3.44it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.45it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.52it/s]\u001b[A\n",
      "  4%|▍         | 1/25 [00:07<02:48,  7.01s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.75it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.69it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.63it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.62it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.60it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.59it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.58it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.58it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.58it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.57it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.57it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.56it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.63it/s]\u001b[A\n",
      "  8%|▊         | 2/25 [00:13<02:40,  6.98s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.72it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.66it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.58it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.56it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.56it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.56it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.55it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.62it/s]\u001b[A\n",
      " 12%|█▏        | 3/25 [00:20<02:32,  6.94s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.82it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.70it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.64it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.61it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.60it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.59it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.58it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.58it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.56it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.63it/s]\u001b[A\n",
      " 16%|█▌        | 4/25 [00:27<02:25,  6.93s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.74it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.67it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.60it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.56it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.56it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.56it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.55it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.62it/s]\u001b[A\n",
      " 20%|██        | 5/25 [00:35<02:20,  7.00s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.71it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.64it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.57it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.55it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.54it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.54it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.54it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.54it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.54it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.61it/s]\u001b[A\n",
      " 24%|██▍       | 6/25 [00:42<02:14,  7.10s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.71it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.64it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.57it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\n",
      " 28%|██▊       | 7/25 [00:49<02:07,  7.11s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.74it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.65it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.58it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.54it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.54it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.54it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.54it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\n",
      " 32%|███▏      | 8/25 [00:57<02:01,  7.16s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.66it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 36%|███▌      | 9/25 [01:04<01:54,  7.14s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.62it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 40%|████      | 10/25 [01:11<01:46,  7.13s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.71it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.63it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.57it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 44%|████▍     | 11/25 [01:18<01:39,  7.12s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.64it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 48%|████▊     | 12/25 [01:25<01:33,  7.16s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.61it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 52%|█████▏    | 13/25 [01:33<01:25,  7.17s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.68it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 56%|█████▌    | 14/25 [01:40<01:19,  7.19s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.60it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.54it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.52it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.51it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.50it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.50it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.50it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.50it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.57it/s]\u001b[A\n",
      " 60%|██████    | 15/25 [01:47<01:11,  7.19s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.70it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.63it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 64%|██████▍   | 16/25 [01:55<01:04,  7.20s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.69it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 68%|██████▊   | 17/25 [02:02<00:57,  7.18s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.66it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.54it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 72%|███████▏  | 18/25 [02:09<00:50,  7.19s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.66it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.53it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.52it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.51it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.50it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.57it/s]\u001b[A\n",
      " 76%|███████▌  | 19/25 [02:16<00:43,  7.18s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.65it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.54it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 80%|████████  | 20/25 [02:23<00:35,  7.17s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.59it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 84%|████████▍ | 21/25 [02:30<00:28,  7.17s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.58it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.53it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.52it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.52it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.50it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.50it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 88%|████████▊ | 22/25 [02:37<00:21,  7.17s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.64it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.53it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.52it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.52it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.51it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.51it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.50it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.50it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.50it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.50it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.50it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.50it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.50it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.50it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.57it/s]\u001b[A\n",
      " 92%|█████████▏| 23/25 [02:44<00:14,  7.16s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.72it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 96%|█████████▌| 24/25 [02:51<00:07,  7.15s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.63it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      "100%|██████████| 25/25 [02:58<00:00,  7.15s/it]\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('yelp_all_model_prediction_ref1.csv', header = None)\n",
    "label = 0\n",
    "label_str = '__label__0'\n",
    "\n",
    "list_sentences = df[1:len(df)].values.tolist()\n",
    "\n",
    "list_sentences_source = []\n",
    "list_sentences_human = []\n",
    "for list_sentance in list_sentences:\n",
    "    list_sentences_source.append(list_sentance[0])\n",
    "    list_sentences_human.append(list_sentance[-1])\n",
    "\n",
    "matrics1 = []\n",
    "for i in tqdm(range(0, len(list_sentences[0]))):\n",
    "    bleu_s = 0\n",
    "    bleu_r = 0\n",
    "    fasttext_c = 0\n",
    "    kenlm_ppl = 0\n",
    "    gpt_ppl = 0\n",
    "    gpt2_ppl = 0\n",
    "    #fastai_c = 0\n",
    "    #fastai_ppl = 0\n",
    "    \n",
    "    sentences = []\n",
    "    for j in range(0, len(list_sentences)):\n",
    "        sentences.append(list_sentences[j][i])\n",
    "        \n",
    "    fasttext_labels = classifier_model.predict(sentences)\n",
    "    \n",
    "    total_sentences = len(sentences)\n",
    "\n",
    "    bleu_s = get_bleu(list_sentences_source, sentences)\n",
    "    bleu_r = get_bleu(list_sentences_human, sentences)\n",
    "\n",
    "    for _, sentence in enumerate(sentences):\n",
    "#         bleu_s += sentence_bleu([list_sentences_source[_]], sentence)\n",
    "#         bleu_r += sentence_bleu([list_sentences_human[_]], sentence)\n",
    "        \n",
    "        if(fasttext_labels[0][_][0]==label_str):\n",
    "            fasttext_c += 1\n",
    "        kenlm_ppl += kenlm_lm.perplexity(sentence)\n",
    "        \n",
    "    labels_list = [label] * len(sentences)\n",
    "    bert_accuracy, pred_label_list = evaluate_dev_set(sentences, labels_list)\n",
    "    \n",
    "    ppl_list_gpt2 = calculate_ppl_gpt2(sentences)\n",
    "    \n",
    "#     for j in range(0, len(ppl_list_gpt1)):\n",
    "#         gpt_ppl += ppl_list_gpt1[j]\n",
    "    for j in range(0, len(ppl_list_gpt2)):\n",
    "        gpt2_ppl += ppl_list_gpt2[j]\n",
    "        \n",
    "#         fastai_label = fastai_classifier.predict(sentence)[1].item()\n",
    "#         if(fastai_label == label):\n",
    "#             fastai_c += 1\n",
    "#         fastai_ppl += perplexitylm(sentence)\n",
    "    matrics1.append([bleu_s , bleu_r , fasttext_c/total_sentences , kenlm_ppl/total_sentences, bert_accuracy, gpt2_ppl/len(ppl_list_gpt2)])\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[100.0,\n",
       "  58.76047151776541,\n",
       "  0.052,\n",
       "  63.416433129944586,\n",
       "  0.008,\n",
       "  22.210357120990754],\n",
       " [49.842279939134116,\n",
       "  35.59684165362755,\n",
       "  0.76,\n",
       "  76.13768818849276,\n",
       "  0.708,\n",
       "  72.25274747371674],\n",
       " [81.05700836941423,\n",
       "  48.870981193551025,\n",
       "  0.078,\n",
       "  89.72583258482746,\n",
       "  0.078,\n",
       "  110.92477409696579],\n",
       " [59.56135976022242,\n",
       "  40.978338773986565,\n",
       "  0.472,\n",
       "  163.29706247774084,\n",
       "  0.42,\n",
       "  216.50014221334456],\n",
       " [23.64798342833085,\n",
       "  18.964238104842654,\n",
       "  0.942,\n",
       "  7.864442458850198,\n",
       "  0.966,\n",
       "  38.179245446681975],\n",
       " [15.161083237506915,\n",
       "  13.302010626509182,\n",
       "  0.158,\n",
       "  246.93935677608124,\n",
       "  0.14,\n",
       "  117.38152573871612],\n",
       " [55.70282939740121,\n",
       "  38.44826753017345,\n",
       "  0.87,\n",
       "  101.97370205084117,\n",
       "  0.894,\n",
       "  71.80047267532349],\n",
       " [59.495961266331896,\n",
       "  41.03778516267212,\n",
       "  0.88,\n",
       "  176.07347929461463,\n",
       "  0.89,\n",
       "  136.54219307804107],\n",
       " [70.09340434198477,\n",
       "  52.9945021277317,\n",
       "  0.9,\n",
       "  81.74441315516009,\n",
       "  0.956,\n",
       "  31.26398136138916],\n",
       " [87.48103866259444,\n",
       "  58.77862094962845,\n",
       "  0.542,\n",
       "  76.77049880755,\n",
       "  0.498,\n",
       "  27.26058078622818],\n",
       " [70.71006476905083,\n",
       "  49.42216530890736,\n",
       "  0.432,\n",
       "  127.82366577900062,\n",
       "  0.462,\n",
       "  55.94305122995377],\n",
       " [77.87716201387093,\n",
       "  51.04520362562218,\n",
       "  0.412,\n",
       "  176.09609632750045,\n",
       "  0.378,\n",
       "  57.43753179645538],\n",
       " [70.07462648012665,\n",
       "  51.60172911880615,\n",
       "  0.784,\n",
       "  115.78340823580194,\n",
       "  0.826,\n",
       "  53.70304404640198],\n",
       " [83.1818220288952,\n",
       "  56.22920378659363,\n",
       "  0.634,\n",
       "  120.97018688386339,\n",
       "  0.612,\n",
       "  58.28040645933151],\n",
       " [68.05241362286121,\n",
       "  51.3176944408941,\n",
       "  0.788,\n",
       "  170.98701188224788,\n",
       "  0.762,\n",
       "  77.45094323682785],\n",
       " [83.55059023862948,\n",
       "  56.907538870432376,\n",
       "  0.6,\n",
       "  117.97383964356467,\n",
       "  0.55,\n",
       "  60.79720264148712],\n",
       " [68.02336587498962,\n",
       "  50.49992128675827,\n",
       "  0.734,\n",
       "  143.27567305902423,\n",
       "  0.772,\n",
       "  81.66808155345917],\n",
       " [85.07727760397592,\n",
       "  56.30511148175694,\n",
       "  0.484,\n",
       "  102.23960798190686,\n",
       "  0.45,\n",
       "  45.54963150835037],\n",
       " [33.837577035840624,\n",
       "  28.549046859248595,\n",
       "  0.836,\n",
       "  8.373610923176749,\n",
       "  0.884,\n",
       "  20.367376957416536],\n",
       " [36.39070614606946,\n",
       "  31.27458393664128,\n",
       "  0.944,\n",
       "  11.012386592098252,\n",
       "  0.962,\n",
       "  18.53719598197937],\n",
       " [15.44831881130007,\n",
       "  15.17363011996433,\n",
       "  0.956,\n",
       "  13.40637546330587,\n",
       "  0.832,\n",
       "  129.57232377290725],\n",
       " [13.016522911223207,\n",
       "  12.70291987646884,\n",
       "  0.984,\n",
       "  8.995393172847766,\n",
       "  0.992,\n",
       "  20.85673448562622],\n",
       " [68.46949722731583,\n",
       "  48.3795620679459,\n",
       "  0.702,\n",
       "  145.0526731650848,\n",
       "  0.748,\n",
       "  76.74051235580444],\n",
       " [26.943991919338806,\n",
       "  25.21479818540562,\n",
       "  0.926,\n",
       "  47.31937495038163,\n",
       "  0.904,\n",
       "  36.81993413710594],\n",
       " [58.7969712073938, 100.0, 0.77, 2571.222684314339, 0.95, 58.92828138113022]]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrics1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/25 [00:00<?, ?it/s]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:05,  2.80it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:04,  3.01it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:04,  3.13it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.19it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.24it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:03,  3.29it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:02<00:02,  3.32it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.35it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:02,  3.37it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.39it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.40it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.42it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.43it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:04<00:00,  3.44it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.45it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.52it/s]\u001b[A\n",
      "  4%|▍         | 1/25 [00:07<02:49,  7.07s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.68it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.65it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.58it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.57it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.56it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.56it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.63it/s]\u001b[A\n",
      "  8%|▊         | 2/25 [00:13<02:39,  6.94s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.75it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.67it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.60it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.59it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.58it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.58it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.57it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.57it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.56it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.63it/s]\u001b[A\n",
      " 12%|█▏        | 3/25 [00:20<02:32,  6.93s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.76it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.70it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.60it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.58it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.57it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.56it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.56it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.63it/s]\u001b[A\n",
      " 16%|█▌        | 4/25 [00:27<02:25,  6.92s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.75it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.67it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.58it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.58it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.57it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.56it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.63it/s]\u001b[A\n",
      " 20%|██        | 5/25 [00:34<02:19,  6.99s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.80it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.68it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.63it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.61it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.60it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.59it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.58it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.57it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.57it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.56it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.56it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.55it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.62it/s]\u001b[A\n",
      " 24%|██▍       | 6/25 [00:42<02:14,  7.08s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.69it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.64it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.58it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.57it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.56it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.56it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.55it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.55it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.55it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.55it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.62it/s]\u001b[A\n",
      " 28%|██▊       | 7/25 [00:49<02:07,  7.10s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.77it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.67it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.59it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.58it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.56it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.56it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.55it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.55it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.54it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.54it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.61it/s]\u001b[A\n",
      " 32%|███▏      | 8/25 [00:56<02:01,  7.12s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.61it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.57it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.55it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.55it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.55it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.54it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.54it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.54it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.54it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.54it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.61it/s]\u001b[A\n",
      " 36%|███▌      | 9/25 [01:03<01:53,  7.10s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.84it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.66it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.58it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.58it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.57it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.56it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.56it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.56it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.55it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.55it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.55it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.54it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.61it/s]\u001b[A\n",
      " 40%|████      | 10/25 [01:10<01:46,  7.09s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.64it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\n",
      " 44%|████▍     | 11/25 [01:18<01:39,  7.10s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.65it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.61it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.55it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\n",
      " 48%|████▊     | 12/25 [01:25<01:32,  7.14s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.59it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.59it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\n",
      " 52%|█████▏    | 13/25 [01:32<01:25,  7.12s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.79it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.65it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.60it/s]\u001b[A\n",
      " 56%|█████▌    | 14/25 [01:39<01:18,  7.13s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.73it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.52it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 60%|██████    | 15/25 [01:46<01:11,  7.12s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.59it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 64%|██████▍   | 16/25 [01:54<01:04,  7.14s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.71it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.62it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 68%|██████▊   | 17/25 [02:01<00:57,  7.14s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.59it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.54it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.52it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 72%|███████▏  | 18/25 [02:08<00:50,  7.15s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.61it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.54it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 76%|███████▌  | 19/25 [02:15<00:42,  7.14s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.73it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.63it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 80%|████████  | 20/25 [02:22<00:35,  7.13s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.79it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.64it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.52it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 84%|████████▍ | 21/25 [02:29<00:28,  7.12s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.60it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.55it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.53it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.52it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.51it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.51it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.51it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.51it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      " 88%|████████▊ | 22/25 [02:36<00:21,  7.11s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:03,  3.80it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.65it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.56it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.54it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.54it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.53it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.53it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 92%|█████████▏| 23/25 [02:43<00:14,  7.11s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.68it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.60it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.57it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.54it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.53it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.52it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.59it/s]\u001b[A\n",
      " 96%|█████████▌| 24/25 [02:50<00:07,  7.10s/it]\n",
      "  0%|          | 0/16 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 1/16 [00:00<00:04,  3.61it/s]\u001b[A\n",
      " 12%|█▎        | 2/16 [00:00<00:03,  3.58it/s]\u001b[A\n",
      " 19%|█▉        | 3/16 [00:00<00:03,  3.56it/s]\u001b[A\n",
      " 25%|██▌       | 4/16 [00:01<00:03,  3.55it/s]\u001b[A\n",
      " 31%|███▏      | 5/16 [00:01<00:03,  3.54it/s]\u001b[A\n",
      " 38%|███▊      | 6/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 44%|████▍     | 7/16 [00:01<00:02,  3.53it/s]\u001b[A\n",
      " 50%|█████     | 8/16 [00:02<00:02,  3.53it/s]\u001b[A\n",
      " 56%|█████▋    | 9/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 62%|██████▎   | 10/16 [00:02<00:01,  3.52it/s]\u001b[A\n",
      " 69%|██████▉   | 11/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 75%|███████▌  | 12/16 [00:03<00:01,  3.52it/s]\u001b[A\n",
      " 81%|████████▏ | 13/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 88%|████████▊ | 14/16 [00:03<00:00,  3.51it/s]\u001b[A\n",
      " 94%|█████████▍| 15/16 [00:04<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 16/16 [00:04<00:00,  3.58it/s]\u001b[A\n",
      "100%|██████████| 25/25 [02:57<00:00,  7.11s/it]\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('yelp_all_model_prediction_ref0.csv', header = None)\n",
    "label = 1\n",
    "label_str = '__label__1'\n",
    "\n",
    "list_sentences = df[1:len(df)].values.tolist()\n",
    "\n",
    "list_sentences_source = []\n",
    "list_sentences_human = []\n",
    "for list_sentance in list_sentences:\n",
    "    list_sentences_source.append(list_sentance[0])\n",
    "    list_sentences_human.append(list_sentance[-1])\n",
    "\n",
    "matrics0 = []\n",
    "for i in tqdm(range(0, len(list_sentences[0]))):\n",
    "    bleu_s = 0\n",
    "    bleu_r = 0\n",
    "    fasttext_c = 0\n",
    "    kenlm_ppl = 0\n",
    "    #gpt_ppl = 0\n",
    "    gpt2_ppl = 0\n",
    "    #fastai_c = 0\n",
    "    #fastai_ppl = 0\n",
    "    \n",
    "    sentences = []\n",
    "    for j in range(0, len(list_sentences)):\n",
    "        sentences.append(list_sentences[j][i])\n",
    "        \n",
    "    fasttext_labels = classifier_model.predict(sentences)\n",
    "    \n",
    "    total_sentences = len(sentences)\n",
    "    \n",
    "    bleu_s = get_bleu(list_sentences_source, sentences)\n",
    "    bleu_r = get_bleu(list_sentences_human, sentences)\n",
    "    \n",
    "    for _, sentence in enumerate(sentences):\n",
    "#         bleu_s += sentence_bleu([list_sentences_source[_]], sentence)\n",
    "#         bleu_r += sentence_bleu([list_sentences_human[_]], sentence)\n",
    "        if(fasttext_labels[0][_][0]==label_str):\n",
    "            fasttext_c += 1\n",
    "        kenlm_ppl += kenlm_lm.perplexity(sentence)\n",
    "        \n",
    "    labels_list = [label] * len(sentences)\n",
    "    bert_accuracy, pred_label_list = evaluate_dev_set(sentences, labels_list)\n",
    "    \n",
    "#     ppl_list_gpt1 = calculate_ppl_gpt(sentences, sentiment=\"<POS>\")\n",
    "    ppl_list_gpt2 = calculate_ppl_gpt2(sentences)\n",
    "    \n",
    "#     for j in range(0, len(ppl_list_gpt1)):\n",
    "#         gpt_ppl += ppl_list_gpt1[j]\n",
    "    for j in range(0, len(ppl_list_gpt2)):\n",
    "        gpt2_ppl += ppl_list_gpt2[j]\n",
    "        \n",
    "#         fastai_label = fastai_classifier.predict(sentence)[1].item()\n",
    "#         if(fastai_label == label):\n",
    "#             fastai_c += 1\n",
    "#         fastai_ppl += perplexitylm(sentence)\n",
    "    matrics0.append([bleu_s , bleu_r , fasttext_c/total_sentences , kenlm_ppl/total_sentences, bert_accuracy, gpt2_ppl/len(ppl_list_gpt2)])\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100.0, 57.333733790819196, 0.038, 82.70006055162344, 0.018, 25.718697894096376]\n",
      "[46.06305753940541, 35.33115113972936, 0.776, 65.26673933836403, 0.754, 73.3109280166626]\n",
      "[74.89798211161994, 45.43406373618434, 0.112, 121.80164262862935, 0.086, 120.84426799345016]\n",
      "[54.95903979143561, 37.19520640834666, 0.54, 155.16837282553405, 0.522, 194.67919037866594]\n",
      "[28.074403766790894, 21.831363013918125, 0.9, 8.633996946904736, 0.96, 39.437523376464846]\n",
      "[68.64257406970076, 44.89914441052, 0.842, 246.93935677608124, 0.86, 117.38152573871612]\n",
      "[57.659395398845184, 38.326106429498466, 0.848, 98.36773751385932, 0.816, 79.8402370018959]\n",
      "[56.46444244003326, 37.83799796003757, 0.876, 61.34533349825818, 0.898, 43.45983244132996]\n",
      "[71.96655090402137, 50.98927581200577, 0.802, 113.42534715416198, 0.822, 45.99616223359108]\n",
      "[89.96628493958603, 55.7498597309706, 0.328, 124.7409502603151, 0.3, 35.14967751455307]\n",
      "[70.16107205739775, 49.576709189366305, 0.73, 171.04383920565138, 0.782, 86.66571102786064]\n",
      "[81.35080373039246, 50.36433177431996, 0.34, 173.87461112423472, 0.32, 58.509598643302915]\n",
      "[71.12837626662972, 52.35271827978784, 0.778, 142.77519543778422, 0.796, 75.1272162334919]\n",
      "[85.33482854015051, 53.072237823629465, 0.502, 158.23907725776544, 0.406, 59.463400008678434]\n",
      "[69.77083909972352, 50.424590745396166, 0.81, 173.50609655664002, 0.834, 89.9697210841179]\n",
      "[86.17035618126911, 53.91123176976039, 0.446, 158.82532225294065, 0.394, 60.12788244724274]\n",
      "[69.69544605973245, 50.49913310508375, 0.774, 173.3930799195574, 0.846, 106.64544916296005]\n",
      "[82.63808291128991, 52.06122049106493, 0.54, 155.96492352301198, 0.476, 54.257784843444824]\n",
      "[29.978369140212752, 29.302166276919138, 0.864, 9.263836801588393, 0.958, 17.887793364524843]\n",
      "[33.44824322381827, 29.391996449096414, 0.954, 11.782223179846305, 0.976, 16.279971998214723]\n",
      "[16.652244654589165, 16.702894119365826, 0.994, 9.239689042748418, 1.0, 12.341705592393875]\n",
      "[12.038123579168149, 12.782445380754984, 0.974, 9.325035285188429, 0.99, 17.93672115588188]\n",
      "[69.42493095174666, 47.75057700539746, 0.824, 179.48599847311496, 0.874, 102.17096532917023]\n",
      "[24.782692480445622, 22.89355010133222, 0.984, 22.69284113820315, 0.99, 20.107868929862978]\n",
      "[57.49862762566356, 100.0, 0.574, 3787.2020141032967, 0.648, 75.56138896799088]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[print(i) for i in matrics0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100.0, 58.76047151776541, 0.052, 63.416433129944586, 0.008, 22.210357120990754]\n",
      "[49.842279939134116, 35.59684165362755, 0.76, 76.13768818849276, 0.708, 72.25274747371674]\n",
      "[81.05700836941423, 48.870981193551025, 0.078, 89.72583258482746, 0.078, 110.92477409696579]\n",
      "[59.56135976022242, 40.978338773986565, 0.472, 163.29706247774084, 0.42, 216.50014221334456]\n",
      "[23.64798342833085, 18.964238104842654, 0.942, 7.864442458850198, 0.966, 38.179245446681975]\n",
      "[15.161083237506915, 13.302010626509182, 0.158, 246.93935677608124, 0.14, 117.38152573871612]\n",
      "[55.70282939740121, 38.44826753017345, 0.87, 101.97370205084117, 0.894, 71.80047267532349]\n",
      "[59.495961266331896, 41.03778516267212, 0.88, 176.07347929461463, 0.89, 136.54219307804107]\n",
      "[70.09340434198477, 52.9945021277317, 0.9, 81.74441315516009, 0.956, 31.26398136138916]\n",
      "[87.48103866259444, 58.77862094962845, 0.542, 76.77049880755, 0.498, 27.26058078622818]\n",
      "[70.71006476905083, 49.42216530890736, 0.432, 127.82366577900062, 0.462, 55.94305122995377]\n",
      "[77.87716201387093, 51.04520362562218, 0.412, 176.09609632750045, 0.378, 57.43753179645538]\n",
      "[70.07462648012665, 51.60172911880615, 0.784, 115.78340823580194, 0.826, 53.70304404640198]\n",
      "[83.1818220288952, 56.22920378659363, 0.634, 120.97018688386339, 0.612, 58.28040645933151]\n",
      "[68.05241362286121, 51.3176944408941, 0.788, 170.98701188224788, 0.762, 77.45094323682785]\n",
      "[83.55059023862948, 56.907538870432376, 0.6, 117.97383964356467, 0.55, 60.79720264148712]\n",
      "[68.02336587498962, 50.49992128675827, 0.734, 143.27567305902423, 0.772, 81.66808155345917]\n",
      "[85.07727760397592, 56.30511148175694, 0.484, 102.23960798190686, 0.45, 45.54963150835037]\n",
      "[33.837577035840624, 28.549046859248595, 0.836, 8.373610923176749, 0.884, 20.367376957416536]\n",
      "[36.39070614606946, 31.27458393664128, 0.944, 11.012386592098252, 0.962, 18.53719598197937]\n",
      "[15.44831881130007, 15.17363011996433, 0.956, 13.40637546330587, 0.832, 129.57232377290725]\n",
      "[13.016522911223207, 12.70291987646884, 0.984, 8.995393172847766, 0.992, 20.85673448562622]\n",
      "[68.46949722731583, 48.3795620679459, 0.702, 145.0526731650848, 0.748, 76.74051235580444]\n",
      "[26.943991919338806, 25.21479818540562, 0.926, 47.31937495038163, 0.904, 36.81993413710594]\n",
      "[58.7969712073938, 100.0, 0.77, 2571.222684314339, 0.95, 58.92828138113022]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[print(i) for i in matrics1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "matricsavg = (np.array(matrics0)+np.array(matrics1))/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "matricsavg = matricsavg.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res0 = pd.DataFrame(matrics0, columns=['BLEU_source','BLEU_human','fasttext_classifier','klm_ppl', 'BERT_classifier', 'gpt2_ppl'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res1 = pd.DataFrame(matrics1, columns=['BLEU_source','BLEU_human','fasttext_classifier','klm_ppl', 'BERT_classifier', 'gpt2_ppl'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_resavg = pd.DataFrame(matricsavg, columns=['BLEU_source','BLEU_human','fasttext_classifier','klm_ppl', 'BERT_classifier', 'gpt2_ppl'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gleu_list = [0.076314, 0.044249, 0.059500, 0.047856, 0.008023, 0.098865, 0.063792, 0.071165, 0.116518, 0.098625, 1.000000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# openaigpt_ppl0 = \n",
    "# openaigpt_ppl1 = \n",
    "\n",
    "# bertclassifier0 = \n",
    "# bertclassifier1 = \n",
    "\n",
    "# gleu0 =\n",
    "# gleu1 =\n",
    "\n",
    "# #df_res.insert(loc=0, column='GLEU_score', value=gleu_list)\n",
    "# df_res0.insert(column='openaigpt_ppl', value=models_list[0])\n",
    "# df_res1.insert(column='openaigpt_ppl', value=models_list[0])\n",
    "# df_resavg.insert(column='openaigpt_ppl', value=models_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "models_list = df[0:1].values.tolist()\n",
    "#df_res.insert(loc=0, column='GLEU_score', value=gleu_list)\n",
    "df_res0.insert(loc=0, column='model', value=models_list[0])\n",
    "df_res1.insert(loc=0, column='model', value=models_list[0])\n",
    "df_resavg.insert(loc=0, column='model', value=models_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>BLEU_source</th>\n",
       "      <th>BLEU_human</th>\n",
       "      <th>fasttext_classifier</th>\n",
       "      <th>klm_ppl</th>\n",
       "      <th>BERT_classifier</th>\n",
       "      <th>gpt2_ppl</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Source</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>58.047103</td>\n",
       "      <td>0.045</td>\n",
       "      <td>73.058247</td>\n",
       "      <td>0.013</td>\n",
       "      <td>23.964528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CROSSALIGNED</td>\n",
       "      <td>47.952669</td>\n",
       "      <td>35.463996</td>\n",
       "      <td>0.768</td>\n",
       "      <td>70.702214</td>\n",
       "      <td>0.731</td>\n",
       "      <td>72.781838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>STYLEEMBEDDING</td>\n",
       "      <td>77.977495</td>\n",
       "      <td>47.152522</td>\n",
       "      <td>0.095</td>\n",
       "      <td>105.763738</td>\n",
       "      <td>0.082</td>\n",
       "      <td>115.884521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MULTIDECODER</td>\n",
       "      <td>57.260200</td>\n",
       "      <td>39.086773</td>\n",
       "      <td>0.506</td>\n",
       "      <td>159.232718</td>\n",
       "      <td>0.471</td>\n",
       "      <td>205.589666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RETRIEVEONLY</td>\n",
       "      <td>25.861194</td>\n",
       "      <td>20.397801</td>\n",
       "      <td>0.921</td>\n",
       "      <td>8.249220</td>\n",
       "      <td>0.963</td>\n",
       "      <td>38.808384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>TEMPLATEBASED</td>\n",
       "      <td>41.901829</td>\n",
       "      <td>29.100578</td>\n",
       "      <td>0.500</td>\n",
       "      <td>246.939357</td>\n",
       "      <td>0.500</td>\n",
       "      <td>117.381526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>DELETEONLY</td>\n",
       "      <td>56.681112</td>\n",
       "      <td>38.387187</td>\n",
       "      <td>0.859</td>\n",
       "      <td>100.170720</td>\n",
       "      <td>0.855</td>\n",
       "      <td>75.820355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>DELETEANDRETRIEVE</td>\n",
       "      <td>57.980202</td>\n",
       "      <td>39.437892</td>\n",
       "      <td>0.878</td>\n",
       "      <td>118.709406</td>\n",
       "      <td>0.894</td>\n",
       "      <td>90.001013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>BERT_DEL</td>\n",
       "      <td>71.029978</td>\n",
       "      <td>51.991889</td>\n",
       "      <td>0.851</td>\n",
       "      <td>97.584880</td>\n",
       "      <td>0.889</td>\n",
       "      <td>38.630072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>SEL_DEL</td>\n",
       "      <td>88.723662</td>\n",
       "      <td>57.264240</td>\n",
       "      <td>0.435</td>\n",
       "      <td>100.755725</td>\n",
       "      <td>0.399</td>\n",
       "      <td>31.205129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>BERT_RET_USE</td>\n",
       "      <td>70.435568</td>\n",
       "      <td>49.499437</td>\n",
       "      <td>0.581</td>\n",
       "      <td>149.433752</td>\n",
       "      <td>0.622</td>\n",
       "      <td>71.304381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>SAL_RET_USE</td>\n",
       "      <td>79.613983</td>\n",
       "      <td>50.704768</td>\n",
       "      <td>0.376</td>\n",
       "      <td>174.985354</td>\n",
       "      <td>0.349</td>\n",
       "      <td>57.973565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>BERT_RET_TFIDF</td>\n",
       "      <td>70.601501</td>\n",
       "      <td>51.977224</td>\n",
       "      <td>0.781</td>\n",
       "      <td>129.279302</td>\n",
       "      <td>0.811</td>\n",
       "      <td>64.415130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>SAL_RET_TFIDF</td>\n",
       "      <td>84.258325</td>\n",
       "      <td>54.650721</td>\n",
       "      <td>0.568</td>\n",
       "      <td>139.604632</td>\n",
       "      <td>0.509</td>\n",
       "      <td>58.871903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>BERT_RET_GLOVE</td>\n",
       "      <td>68.911626</td>\n",
       "      <td>50.871143</td>\n",
       "      <td>0.799</td>\n",
       "      <td>172.246554</td>\n",
       "      <td>0.798</td>\n",
       "      <td>83.710332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>SAL_RET_GLOVE</td>\n",
       "      <td>84.860473</td>\n",
       "      <td>55.409385</td>\n",
       "      <td>0.523</td>\n",
       "      <td>138.399581</td>\n",
       "      <td>0.472</td>\n",
       "      <td>60.462543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>BERT_RET_RANDOM</td>\n",
       "      <td>68.859406</td>\n",
       "      <td>50.499527</td>\n",
       "      <td>0.754</td>\n",
       "      <td>158.334376</td>\n",
       "      <td>0.809</td>\n",
       "      <td>94.156765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>SAL_RET_RANDOM</td>\n",
       "      <td>83.857680</td>\n",
       "      <td>54.183166</td>\n",
       "      <td>0.512</td>\n",
       "      <td>129.102266</td>\n",
       "      <td>0.463</td>\n",
       "      <td>49.903708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>RET_ONLY_USE</td>\n",
       "      <td>31.907973</td>\n",
       "      <td>28.925607</td>\n",
       "      <td>0.850</td>\n",
       "      <td>8.818724</td>\n",
       "      <td>0.921</td>\n",
       "      <td>19.127585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>RET_ONLY_TFIDF</td>\n",
       "      <td>34.919475</td>\n",
       "      <td>30.333290</td>\n",
       "      <td>0.949</td>\n",
       "      <td>11.397305</td>\n",
       "      <td>0.969</td>\n",
       "      <td>17.408584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>RET_ONLY_GLOVE</td>\n",
       "      <td>16.050282</td>\n",
       "      <td>15.938262</td>\n",
       "      <td>0.975</td>\n",
       "      <td>11.323032</td>\n",
       "      <td>0.916</td>\n",
       "      <td>70.957015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>RET_ONLY_RANDOM</td>\n",
       "      <td>12.527323</td>\n",
       "      <td>12.742683</td>\n",
       "      <td>0.979</td>\n",
       "      <td>9.160214</td>\n",
       "      <td>0.991</td>\n",
       "      <td>19.396728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>BERT_STS_RET</td>\n",
       "      <td>68.947214</td>\n",
       "      <td>48.065070</td>\n",
       "      <td>0.763</td>\n",
       "      <td>162.269336</td>\n",
       "      <td>0.811</td>\n",
       "      <td>89.455739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>BACK_TRANS</td>\n",
       "      <td>25.863342</td>\n",
       "      <td>24.054174</td>\n",
       "      <td>0.955</td>\n",
       "      <td>35.006108</td>\n",
       "      <td>0.947</td>\n",
       "      <td>28.463902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>HUMAN</td>\n",
       "      <td>58.147799</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.672</td>\n",
       "      <td>3179.212349</td>\n",
       "      <td>0.799</td>\n",
       "      <td>67.244835</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                model  BLEU_source  BLEU_human  fasttext_classifier  \\\n",
       "0              Source   100.000000   58.047103                0.045   \n",
       "1        CROSSALIGNED    47.952669   35.463996                0.768   \n",
       "2      STYLEEMBEDDING    77.977495   47.152522                0.095   \n",
       "3        MULTIDECODER    57.260200   39.086773                0.506   \n",
       "4        RETRIEVEONLY    25.861194   20.397801                0.921   \n",
       "5       TEMPLATEBASED    41.901829   29.100578                0.500   \n",
       "6          DELETEONLY    56.681112   38.387187                0.859   \n",
       "7   DELETEANDRETRIEVE    57.980202   39.437892                0.878   \n",
       "8            BERT_DEL    71.029978   51.991889                0.851   \n",
       "9             SEL_DEL    88.723662   57.264240                0.435   \n",
       "10       BERT_RET_USE    70.435568   49.499437                0.581   \n",
       "11        SAL_RET_USE    79.613983   50.704768                0.376   \n",
       "12     BERT_RET_TFIDF    70.601501   51.977224                0.781   \n",
       "13      SAL_RET_TFIDF    84.258325   54.650721                0.568   \n",
       "14     BERT_RET_GLOVE    68.911626   50.871143                0.799   \n",
       "15      SAL_RET_GLOVE    84.860473   55.409385                0.523   \n",
       "16    BERT_RET_RANDOM    68.859406   50.499527                0.754   \n",
       "17     SAL_RET_RANDOM    83.857680   54.183166                0.512   \n",
       "18       RET_ONLY_USE    31.907973   28.925607                0.850   \n",
       "19     RET_ONLY_TFIDF    34.919475   30.333290                0.949   \n",
       "20     RET_ONLY_GLOVE    16.050282   15.938262                0.975   \n",
       "21    RET_ONLY_RANDOM    12.527323   12.742683                0.979   \n",
       "22       BERT_STS_RET    68.947214   48.065070                0.763   \n",
       "23         BACK_TRANS    25.863342   24.054174                0.955   \n",
       "24              HUMAN    58.147799  100.000000                0.672   \n",
       "\n",
       "        klm_ppl  BERT_classifier    gpt2_ppl  \n",
       "0     73.058247            0.013   23.964528  \n",
       "1     70.702214            0.731   72.781838  \n",
       "2    105.763738            0.082  115.884521  \n",
       "3    159.232718            0.471  205.589666  \n",
       "4      8.249220            0.963   38.808384  \n",
       "5    246.939357            0.500  117.381526  \n",
       "6    100.170720            0.855   75.820355  \n",
       "7    118.709406            0.894   90.001013  \n",
       "8     97.584880            0.889   38.630072  \n",
       "9    100.755725            0.399   31.205129  \n",
       "10   149.433752            0.622   71.304381  \n",
       "11   174.985354            0.349   57.973565  \n",
       "12   129.279302            0.811   64.415130  \n",
       "13   139.604632            0.509   58.871903  \n",
       "14   172.246554            0.798   83.710332  \n",
       "15   138.399581            0.472   60.462543  \n",
       "16   158.334376            0.809   94.156765  \n",
       "17   129.102266            0.463   49.903708  \n",
       "18     8.818724            0.921   19.127585  \n",
       "19    11.397305            0.969   17.408584  \n",
       "20    11.323032            0.916   70.957015  \n",
       "21     9.160214            0.991   19.396728  \n",
       "22   162.269336            0.811   89.455739  \n",
       "23    35.006108            0.947   28.463902  \n",
       "24  3179.212349            0.799   67.244835  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_resavg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [print(i) for i in df_resavg['BLEU_source'].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res0.to_csv('matrics/yelp/matrics_yelp_all_model_prediction_0.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res1.to_csv('matrics/yelp/matrics_yelp_all_model_prediction_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_resavg.to_csv('matrics/yelp/matrics_yelp_all_model_prediction_avg.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
